package com.gxljc.bear.crawler.itaogao.szsb;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;
import com.gxljc.bear.crawler.util.DateUtil;
import com.gxljc.bear.crawler.util.JsoupUtil;

/**
 * 深圳商报
 * 
 * @author Administrator -> haitao
 * @since 2021年9月26日 上午11:20:44
 */
public class SzsbDetailCrawler extends NewsBaseCrawler {

	/**
	 * 
	 */
	private static final long serialVersionUID = -983583436308296597L;
	private static final Logger LOG = Logger.getLogger(SzsbDetailCrawler.class);
	final static String LIST_FORMAT = "http://szsb.sznews.com/html/%s/node_1388.htm";
	final static String URL_FORMAT = "http://szsb.sznews.com/html/%s/%s";

	public SzsbDetailCrawler() {
		super("t_newspaper_szsb");

	}

	protected List<String> genSeeds() {
		LOG.info("--- genSeed of ccstock ----");
		List<String> seeds = new ArrayList<>();
		try {
			for (String date : genDatesTillNow(DEFAULT_DATE_FORMAT)) {
				// fetch url list by parser
				String url = String.format(LIST_FORMAT, date);
				byte[] bytes = fetch(url, proxy);
				if (bytes == null) {
					continue;
				}
				String html = new String(bytes);
				String css = "tbody tr td table tbody tr td div table tbody tr td table tbody tr td a";
				List<String> hrefList = JsoupUtil.extractAttrList(html, css, "href");
				List<String> urls = new ArrayList<>();
				for (String s : hrefList) {
					if (!s.startsWith("content_")) {
						continue;
					}
					urls.add(String.format(URL_FORMAT, date, s));
				}
				seeds.addAll(urls);
			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	@Override
	protected List<String> genNewSeeds() {
		LOG.info("--- genNewSeed of ccstock ----");
		List<String> seeds = new ArrayList<>();
		String date = DateUtil.dateFormat(new Date(), "yyyy-MM/dd");
		// fetch url list by parser
		String url = String.format(LIST_FORMAT, date);
		String html = "";
		try {
			byte[] bytes = fetch(url, proxy);
			if (bytes == null) {
				LOG.error(String.format("fetch empty for url=", url));
				return seeds;
			}
			html = new String(bytes);
			String css = "tbody tr td table tbody tr td div table tbody tr td table tbody tr td a";
			List<String> hrefList = JsoupUtil.extractAttrList(html, css, "href");
			List<String> urls = new ArrayList<>();
			for (String s : hrefList) {
				if (!s.startsWith("content_")) {
					continue;
				}
				urls.add(String.format(URL_FORMAT, date, s));
			}
			seeds.addAll(urls);
		} catch (Exception e) {
			e.printStackTrace();
		}

		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	@SuppressWarnings("unchecked")
	@Override
	protected Pair<NewsBaseTable, byte[]> genTable(String url) {
		NewsBaseTable table = new NewsBaseTable();
		String request = url;
		String result = "";
		Pair<String, byte[]> pair = fetResult(request, proxy);
		if (pair == null) {
			return new Pair<NewsBaseTable, byte[]>(table, "".getBytes());
		}
		result = pair.first;
		byte[] html = pair.second;
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			if (jsonResult != null) {

				table.setPublishDate(extractDate(url));
				String title = (String) jsonResult.get("title");
				if (!StringUtil.isNullOrEmpty(title)) {
					table.setTitle(title);
				}
				String subTitle = (String) jsonResult.get("sub_title");
				if (!StringUtil.isNullOrEmpty(subTitle)) {
					table.setSubTitle(subTitle);
				}
				String author = (String) jsonResult.get("author");
				if (!StringUtil.isNullOrEmpty(author)) {
					table.setAuthor(author);
				}
				List<String> images = (List<String>) jsonResult.get("pic_list");
				LOG.info("images=" + images);
				if (validateObject(images)) {
					table = processImages(url, table, images);
				}
				List<String> paraList = (List<String>) jsonResult.get("para_list");
				if (paraList != null) {
					table.setContent(genContent(paraList, table.getImages()));
				}
				// set default value
				table.setId(StringUtil.genMD5Val(url));
				table.setCrawlUrl(url);
				table.setCrawlDate(new Date());
				table.setCrawlFlag(1);
			}
		}
		return new Pair<NewsBaseTable, byte[]>(table, html);

	}

//	public static void main(String[] args) {
//		String url = "http://szsb.sznews.com/html/2016-08/19/node_1388.htm";
//		String css = "tbody tr td table tbody tr td div table tbody tr td table tbody tr td a";
//		System.out.println(JsoupUtil.extractAttrListByUrl(url, css, "href"));
//
//		System.out.println("-- start --");
//		String request = "http://epaper.bjbusiness.com.cn/site1/bjsb/html/2016-09/12/node_226.htm";
//		request = "http://szsb.sznews.com/html/2016-08/19/content_3598853.htm";
//		SzsbDetailCrawler crawler = new SzsbDetailCrawler();
//		Pair<String, byte[]> pair = crawler.fetResult(request, false);
//		if (pair != null) {
//			System.out.println(pair.first);
//		}
//	}

}
